# coding:utf-8
from models import meizitu_thread
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import base64

import os
import re
from models import *

from settings import *
import  urllib3
import re
import time

engine = create_engine(DATABASE_URL)
Session = sessionmaker(bind=engine)

session = Session()

from selenium import webdriver


class MeizituCrawer(object):
    base_url = 'http://meizitu.com/a/more_{0}.html'

    def __init__(self):
        self.driver = webdriver.Firefox()
        self.driver.get(self.get_url())
        last_item = self.driver.find_elements_by_css_selector("#wp_page_numbers li a")[-1].get_attribute("href")
        self.last = int(re.search("more_(\d+)\.html", last_item).groups()[0])
        self.init()

    def get_url(self, _id=1):
        return self.base_url.format(_id)

    def init(self):
        for i in range(1, self.last + 1):
            self.update(i)

    def start(self):
        while True:
            self.update(1)
            time.sleep(600)

    def update(self, _id=1):
        print("开始抓取 " + self.get_url(_id))
        self.driver.get(self.get_url(_id))
        items = self.driver.find_elements_by_css_selector(".wp-item")
        for item in items:
            a = item.find_element_by_css_selector(".tit a")
            url = a.get_attribute("href")
            title = a.text.replace("*", "").replace("|", "")
            self.add_thread(url, title)
        self.update_thread()
        print("结束抓取 " + self.get_url(_id))


    def add_thread(self, url, title):
        thread = session.query(meizitu_thread).filter_by(url=url, title=title).first()
        if not thread:
            session.add(meizitu_thread(url=url, title=title, status="new"))
            session.commit()
            print("添加条目  " + title)
        else:
            print("跳过条目 " + title)

    def add_tag(self, thread_id, name):
        tag = session.query(meizitu_tag).filter_by(name=name).first()
        if not tag:
            tag = meizitu_tag(name=name)
            session.add(tag)
            session.commit()
        mapper = session.query(meizitu_thread_tag_mapper).filter_by(meizitu_thread_id=thread_id,
            meizitu_tag_id=tag.id).first()
        if not mapper:
            mapper = meizitu_thread_tag_mapper(meizitu_thread_id=thread_id,
                                               meizitu_tag_id=tag.id)
            session.add(mapper)
            session.commit()

    def update_thread(self):
        threads = session.query(meizitu_thread).filter_by(status="new")
        for thread in threads:
            print("抓取条目 " + thread.title)
            self.driver.get(thread.url)
            tags = self.driver.find_element_by_css_selector(".postmeta p")
            if tags.text.startswith("Tags:"):
                tag_strings = map(lambda x: x.strip(), filter(lambda x: x.strip(), tags.text.split(":")[-1].split(",")))
                for tag in tag_strings:
                    self.add_tag(thread.id, tag)
                day = int(self.driver.find_element_by_css_selector(".day").text)
                month_year = self.driver.find_element_by_css_selector(".month_Year").text
                month, year = month_year.split(" ")
                month, year = int(month), int(year)
                thread.updated_at = int(time.mktime((year, month, day, 0, 0, 0, 0, 0, 0)))
                session.commit()
                for picture in self.driver.find_elements_by_css_selector("#picture img"):
                    url = picture.get_attribute("src")
                    title = picture.get_attribute("alt")
                    self.add_picutre(thread.id, url, title)
                thread.status = "complete"
                session.commit()
                print("结束抓取条目 " + thread.title)
            else:
                print("no tag: ", tags.text)

    def add_picutre(self, thread_id, url, title):
        picture = session.query(meizitu_picture).filter_by(meizitu_thread_id=thread_id,
            url=url, title=title).first()
        if not picture:
            picture = meizitu_picture(meizitu_thread_id=thread_id, url=url, title=title, status="new")
            session.add(picture)
            session.commit()

meizituCrawer = MeizituCrawer()
meizituCrawer.start()
